package com.xiaotu.spider.guduo.pageprocessor;

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.stereotype.Component;

import com.xiaotu.common.model.GuDuoExceptionRecordModel;
import com.xiaotu.common.model.GuDuoPublicOpinionDataModel;
import com.xiaotu.common.model.GuDuoRecordDataModel;
import com.xiaotu.common.model.TvInfoSpiderTempModel;
import com.xiaotu.common.util.CacheHandler;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.DateUtils;
import com.xiaotu.common.util.GsonUtils;
import com.xiaotu.common.util.HttpUtils;
import com.xiaotu.common.util.MathUtil;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.PropertiesUtil;
import com.xiaotu.common.util.UUIDUtils;
import com.xiaotu.server.service.guduo.SupplementGuDuoRecordService;
import com.xiaotu.spider.AUrlPageProcessor;

import us.codecraft.webmagic.Page;

/***
 * 补充骨朵异常爬取信息（将剧基本信息补充到今天）
 * @author 
 *
 */
@Component("SupplementGuDuoExceptionRecordDataPageProcessor")
public class SupplementGuDuoExceptionRecordDataPageProcessor extends AUrlPageProcessor {

	private static String TV_INFO_SPIDER_TEMP = "TV_INFO_SPIDER_TEMP_"+DateUtils.getCurrentDate(new Date(), DateUtils.FILE_NOMENCLATURE_FORMAT);
	
	@Autowired
	private CacheHandler cacheHandler;
	
	//用于去重剧
	private static Map<String, Object> distinctMap = new HashMap<String, Object>();
	
	private String start_url = "https://www.baidu.com/";
	
	@Autowired
	private SupplementGuDuoRecordService supplementGuDuoRecordService;
	
	private static Map<String, String> headers = null;
	
	static {
		headers = new HashMap<String,String>();
	    headers.put("Accept", PropertiesUtil.getProperty("GuDuo.site.Accept"));
	    headers.put("Accept-Encoding", PropertiesUtil.getProperty("GuDuo.site.Accept-Encoding"));
	    headers.put("Accept-Language", PropertiesUtil.getProperty("GuDuo.site.Accept-Language"));
	    headers.put("Cache-Control", PropertiesUtil.getProperty("GuDuo.site.Cache-Control"));
	    headers.put("Connection", PropertiesUtil.getProperty("GuDuo.site.Connection"));
	    headers.put("Cookie", PropertiesUtil.getProperty("GuDuo.site.Cookie"));
	    headers.put("Host", PropertiesUtil.getProperty("GuDuo.site.Host"));
	    headers.put("User-Agent", PropertiesUtil.getProperty("GuDuo.site.User-Agent"));
	}
	
	
	@Override
	public void process(Page page) {
		
		switch (this.getPageDataType(page))
        {
            default:
            	this.addRequestProcess(page);
            	break;
        }
		
	}

	Map<String,Object> distinctDataidMap = new HashMap<String,Object>();
	
	//补充评分相关
	private void simpleOverviewProcess(String url, Map<String, Object> extrasMap) {
		
		List<GuDuoExceptionRecordModel> updExceptionRecordList = new ArrayList<GuDuoExceptionRecordModel>();
		List<GuDuoRecordDataModel> updRecordDataList = new ArrayList<GuDuoRecordDataModel>();
		List<GuDuoPublicOpinionDataModel> addGuDuoPublicOpinionDataList = new ArrayList<GuDuoPublicOpinionDataModel>();
		List<GuDuoPublicOpinionDataModel> updGuDuoPublicOpinionDataList = new ArrayList<GuDuoPublicOpinionDataModel>();
		
		
		GuDuoExceptionRecordModel model = (GuDuoExceptionRecordModel)extrasMap.get("model");
		GuDuoRecordDataModel recordDataModel = (GuDuoRecordDataModel)extrasMap.get("guDuoRecordDataModel");
		
		//豆瓣评分
        String douban_score = null;
        Double douban_scoreLong = null;
		//豆瓣评分人数
        String douban_judges_count = null;
        Double douban_judges_countLong = null;
        //豆瓣短评数
		String doubanShortCommenNum = null;
		Double doubanShortCommenNumLong = null;
		//评论总数
		String totalCommentNum = null;
		Double totalCommentNumLong = null;
		//总弹幕数
		String totalBarrageNum = null;
		Double totalBarrageNumLong = null;
		//总新闻收录数
		String news_count = null;
		Double news_countLong = null;
		//微信总公众号文章数
		String wechat_count = null;
		Double wechat_countLong = null;
		//贴吧总帖子数
		String tieba_count = null;
		Double tieba_countLong = null;
		//总相关微博数
		String weibo_count = null;
		Double weibo_countLong = null;
		//微博总话题数
		String weibo_tag_count = null;
		Double weibo_tag_countLong = null;
		//微博总阅读量
		String weibo_tag_read_count = null;
		Double weibo_tag_read_countLong = null;
		//累计播放量
		String totalPlayCount = null;
		
		
		HttpUtils.TIMEOUT = 60000;
		String sumJSONP = HttpUtils.httpGetJSONP(url,headers);
		
		Map<String, Object> sumMap = GsonUtils.fromJson(sumJSONP, Map.class);
		
		douban_score = sumMap.get("douban_score")==null?null:sumMap.get("douban_score").toString();
        if(StringUtils.isBlank(douban_score.replaceAll("[\u4e00-\u9fa5]", ""))){
        	douban_score = null;
        }
        if(null!=douban_score && StringUtils.isNotBlank(douban_score) 
         		 && MathUtil.isNumber(douban_score)
         		 && MathUtil.isPositiveNumber(douban_score)){
        	douban_scoreLong = Double.parseDouble(douban_score);
   		} 
		//豆瓣评分人数
		douban_judges_count = sumMap.get("douban_judges_count") == null?null:sumMap.get("douban_judges_count").toString();
		if(null!=douban_judges_count && StringUtils.isNotBlank(douban_judges_count) 
      		 && MathUtil.isNumber(douban_judges_count)
      		 && MathUtil.isPositiveNumber(douban_judges_count)){
			douban_judges_countLong = Double.parseDouble(douban_judges_count);
		}
		//豆瓣短评数
		doubanShortCommenNum = sumMap.get("douban_brief_comment_count") == null?null:sumMap.get("douban_brief_comment_count").toString();
		if(null!=doubanShortCommenNum && StringUtils.isNotBlank(doubanShortCommenNum) 
      		 && MathUtil.isNumber(doubanShortCommenNum)
      		 && MathUtil.isPositiveNumber(doubanShortCommenNum)){
			doubanShortCommenNumLong = Double.parseDouble(doubanShortCommenNum);
		}
		
		//评论总数
		totalCommentNum = sumMap.get("comment_count") == null?null:sumMap.get("comment_count").toString();
     
		if(null!=totalCommentNum && StringUtils.isNotBlank(totalCommentNum) 
     		 && MathUtil.isNumber(totalCommentNum)
     		 && MathUtil.isPositiveNumber(totalCommentNum)){
			totalCommentNumLong = Double.parseDouble(totalCommentNum);
		}
		
		//总弹幕数
		totalBarrageNum = sumMap.get("barrage_count") == null?null:sumMap.get("barrage_count").toString();
		
		if(null!=totalBarrageNum && StringUtils.isNotBlank(totalBarrageNum) 
     		 && MathUtil.isNumber(totalBarrageNum)
     		 && MathUtil.isPositiveNumber(totalBarrageNum)){
			totalBarrageNumLong = Double.parseDouble(totalBarrageNum);
		}
		
		//总新闻收录数
		news_count = sumMap.get("news_count") == null?null:sumMap.get("news_count").toString();
		
		if(null!=news_count && StringUtils.isNotBlank(news_count) 
     		 && MathUtil.isNumber(news_count)
     		 && MathUtil.isPositiveNumber(news_count)){
			news_countLong = Double.parseDouble(news_count);
		}
		//微信总公众号文章数
		wechat_count = sumMap.get("wechat_count") == null?null:sumMap.get("wechat_count").toString();
		
		if(null!=wechat_count && StringUtils.isNotBlank(wechat_count) 
     		 && MathUtil.isNumber(wechat_count)
     		 && MathUtil.isPositiveNumber(wechat_count)){
			wechat_countLong = Double.parseDouble(wechat_count);
		}
		//贴吧总帖子数
		tieba_count = sumMap.get("tieba_count") == null?null:sumMap.get("tieba_count").toString();
		
		if(null!=tieba_count && StringUtils.isNotBlank(tieba_count) 
     		 && MathUtil.isNumber(tieba_count)
     		 && MathUtil.isPositiveNumber(tieba_count)){
			tieba_countLong = Double.parseDouble(tieba_count);
		}
		//总相关微博数
		weibo_count = sumMap.get("weibo_count") == null?null:sumMap.get("weibo_count").toString();
		
		if(null!=weibo_count && StringUtils.isNotBlank(weibo_count) 
     		 && MathUtil.isNumber(weibo_count)
     		 && MathUtil.isPositiveNumber(weibo_count)){
			weibo_countLong = Double.parseDouble(weibo_count);
		}
		//微博总话题数
		weibo_tag_count = sumMap.get("weibo_tag_count") == null?null:sumMap.get("weibo_tag_count").toString();
		
		if(null!=weibo_tag_count && StringUtils.isNotBlank(weibo_tag_count) 
     		 && MathUtil.isNumber(weibo_tag_count)
     		 && MathUtil.isPositiveNumber(weibo_tag_count)){
			weibo_tag_countLong = Double.parseDouble(weibo_tag_count);
		}
		//微博总阅读量
		weibo_tag_read_count = sumMap.get("weibo_tag_read_count") == null?null:sumMap.get("weibo_tag_read_count").toString();
		
		if(null!=weibo_tag_read_count && StringUtils.isNotBlank(weibo_tag_read_count) 
     		 && MathUtil.isNumber(weibo_tag_read_count)
     		 && MathUtil.isPositiveNumber(weibo_tag_read_count)){
			weibo_tag_read_countLong = Double.parseDouble(weibo_tag_read_count);
        }
		
		//累计播放量
		totalPlayCount = sumMap.get("play_count_count") == null?null:sumMap.get("play_count_count").toString();
		if(null!=totalPlayCount && StringUtils.isNotBlank(totalPlayCount)){
			totalPlayCount = new BigDecimal(totalPlayCount).toPlainString();
			if(!(MathUtil.isNumber(totalPlayCount)
		    		 && MathUtil.isPositiveNumber(totalPlayCount))){
				totalPlayCount = null;
			}
        }
		
		model.setSupplement_status(1);//已补充
		updExceptionRecordList.add(model);
		//骨朵舆情概况
		GuDuoPublicOpinionDataModel guDuoPublicOpinionDataModel = new GuDuoPublicOpinionDataModel();
		if(!distinctDataidMap.containsKey(model.getDataid())){
			guDuoPublicOpinionDataModel.setResourcesiteid(Constants.SiteName.GUDUO);
			guDuoPublicOpinionDataModel.setTvname(model.getTvname());
			guDuoPublicOpinionDataModel.setDataid(model.getDataid());
			guDuoPublicOpinionDataModel.setDouban_score(douban_scoreLong);
			guDuoPublicOpinionDataModel.setDouban_judges_count(douban_judges_countLong);
			guDuoPublicOpinionDataModel.setDouban_brief_comment_count(doubanShortCommenNumLong);
			guDuoPublicOpinionDataModel.setComment_count(totalCommentNumLong);
			guDuoPublicOpinionDataModel.setBarrage_count(totalBarrageNumLong);
			guDuoPublicOpinionDataModel.setNews_count(news_countLong);
			guDuoPublicOpinionDataModel.setWechat_count(wechat_countLong);
			guDuoPublicOpinionDataModel.setTieba_count(tieba_countLong);
			guDuoPublicOpinionDataModel.setWeibo_count(weibo_countLong);
			guDuoPublicOpinionDataModel.setWeibo_tag_count(weibo_tag_countLong);
			guDuoPublicOpinionDataModel.setWeibo_tag_read_count(weibo_tag_read_countLong);
			guDuoPublicOpinionDataModel.setExceptionCollectTime(model.getException_date());
			if(supplementGuDuoRecordService.getPublicOpinion(model)==0){
				guDuoPublicOpinionDataModel.setId(UUIDUtils.getId());
				addGuDuoPublicOpinionDataList.add(guDuoPublicOpinionDataModel);
			}else{
				guDuoPublicOpinionDataModel.setId(model.getId());
				updGuDuoPublicOpinionDataList.add(guDuoPublicOpinionDataModel);
			}
			distinctDataidMap.put(model.getDataid(), model.getDataid());
		 }
		 
		 //骨朵过程数据（用于修改遗漏的播放平台）
		 GuDuoRecordDataModel guDuoRecordDataModel = new GuDuoRecordDataModel();
		 guDuoRecordDataModel.setExceptionCollectTime(model.getException_date());
		 guDuoRecordDataModel.setDataId(model.getDataid());
		 guDuoRecordDataModel.setTotalPlayCount(totalPlayCount);
		 guDuoRecordDataModel.setPlatformName(model.getPlatform_name());
		 if(null != recordDataModel){
			 guDuoRecordDataModel.setPlatFormNameArr(recordDataModel.getPlatFormNameArr());
		 }
		 updRecordDataList.add(guDuoRecordDataModel);
		 supplementGuDuoRecordService.updateDataList(updExceptionRecordList,updRecordDataList,addGuDuoPublicOpinionDataList,updGuDuoPublicOpinionDataList);
	}

	//补充播放平台及剧的基本信息存入缓存
	@SuppressWarnings("unchecked")
	private void simpleDetailProcess(String url,Map<String, Object> extrasMap) {
		//播放平台
        String [] platFormNameArr = null;
		GuDuoExceptionRecordModel model = (GuDuoExceptionRecordModel)extrasMap.get("model");
		HttpUtils.TIMEOUT = 60000;
		String dataJSONP = HttpUtils.httpGetJSONP(url,headers);
		Map<String, Object> everyMap = GsonUtils.fromJson(dataJSONP, Map.class);
		
		GuDuoRecordDataModel guDuoRecordDataModel = new GuDuoRecordDataModel();
        List<String> platFormImgList =  (List<String>)everyMap.get("platform_img_list");
        if(null != platFormImgList && platFormImgList.size()>0){
        	platFormNameArr = new String[platFormImgList.size()];
            for(int i=0;i<platFormImgList.size();i++){
            	if(platFormImgList.get(i).contains("youku")){
					platFormNameArr[i] = "优酷";
				}else if(platFormImgList.get(i).contains("aiqiyi")){
					platFormNameArr[i] = "爱奇艺";
				}else if(platFormImgList.get(i).contains("souhu")){
					platFormNameArr[i] = "搜狐视频";
				}else if(platFormImgList.get(i).contains("mamgguo")){
					platFormNameArr[i] = "芒果TV";
				}else if(platFormImgList.get(i).contains("tengxun")){
					platFormNameArr[i] = "腾讯视频";
				}else if(platFormImgList.get(i).contains("pjuli")){
					platFormNameArr[i] = "PPTV";
				}
            }
            platFormNameArr = distinctArr(platFormNameArr);
        }
        
        guDuoRecordDataModel.setPlatFormNameArr(platFormNameArr);
        
        //将剧的基本信息存入缓存中用于匹配猫眼剧
        
        Map<String,Object> guoDuoInfoMap = new HashMap<String,Object>(); 
         
        //剧类型
        String movieType = everyMap.get("type")==null?null:everyMap.get("type").toString().replaceAll("[/、]", ",").replaceAll("[ ]", "");
        //导演
        String director = everyMap.get("director")==null?null:everyMap.get("director").toString().replaceAll("[/、]", ",").replaceAll("[ ]", "");
        //演员
        String actors = everyMap.get("actors")==null?null:everyMap.get("actors").toString().replaceAll("[/、]", ",").replaceAll("[ ]", "");
        //编剧
        String screenwriter = everyMap.get("script_writer")==null?null:everyMap.get("script_writer").toString().replaceAll("[/、]", ",").replaceAll("[ ]", "");
        //制片人
        String producer = everyMap.get("producer")==null?null:everyMap.get("producer").toString().replaceAll("[/、]", ",").replaceAll("[ ]", "");
        //出品公司
        String issuecompany = "";
        
        List<String> issuecompanyList =  (List<String>)everyMap.get("publishers");
        if(null != issuecompanyList && issuecompanyList.size()>0){
 	            for(int i=0;i<issuecompanyList.size();i++){
 	            	issuecompany+=","+issuecompanyList.get(i);
 	            }
 	           if(issuecompany.length()>1){
     	        	 issuecompany = issuecompany.substring(1);
     	        }  
        }
        
        //集数
        String sets = everyMap.get("episode")==null?null:everyMap.get("episode").toString().replaceAll("[ ]", "");
        if(null != sets && StringUtils.isNotBlank(sets) && sets.indexOf(".0")!=-1){
      	  sets = sets.replace(".0", "");
        }
        //时长
        String timeLength = everyMap.get("duration")==null?null:everyMap.get("duration").toString().replaceAll("[ ]", "").replace("分钟", "");
        //上映时间
        String relaseDate = everyMap.get("release_date")==null?null:everyMap.get("release_date").toString().replaceAll("[ ]", "");
        
        //图片链接
        String imageurl = everyMap.get("cover")==null?null:everyMap.get("cover").toString();
        
        
        guoDuoInfoMap.put("id", UUIDUtils.getId());
        guoDuoInfoMap.put("tv_dataid", model.getDataid());
        guoDuoInfoMap.put("tvname", model.getTvname());
        guoDuoInfoMap.put("imageurl", imageurl);
        guoDuoInfoMap.put("label", movieType);
        guoDuoInfoMap.put("director", director);
        guoDuoInfoMap.put("actor", actors);
        guoDuoInfoMap.put("screenwriter", screenwriter);
        guoDuoInfoMap.put("producer", producer);
        guoDuoInfoMap.put("issuecompany", issuecompany);
        guoDuoInfoMap.put("sets", sets);
        guoDuoInfoMap.put("timeLength", timeLength);
        guoDuoInfoMap.put("tvtype", model.getTvtype());
        guoDuoInfoMap.put("relasedate", relaseDate);
        guoDuoInfoMap.put("brodcastplatform", platFormNameArr);
        guoDuoInfoMap.put("resourcesiteid", Constants.SiteName.GUDUO);
        guoDuoInfoMap.put("collecttime", System.currentTimeMillis());
        
        if(!cacheHandler.cacheExist(TV_INFO_SPIDER_TEMP)){
          	if(!distinctMap.containsKey(model.getDataid())){
          		TvInfoSpiderTempModel queryisExitsSpiderTemp = supplementGuDuoRecordService.queryisExitsSpiderTemp(guoDuoInfoMap);
  	    		if(null == queryisExitsSpiderTemp){
  	    			cacheHandler.hSet(TV_INFO_SPIDER_TEMP, model.getDataid(), guoDuoInfoMap);
  	    		}
              }
      	    }else{
      	    	Map<String, Object> hGetAll = cacheHandler.hGet(TV_INFO_SPIDER_TEMP, model.getDataid(), Map.class);
      	    	if(null == hGetAll){
      	    		TvInfoSpiderTempModel queryisExitsSpiderTemp = supplementGuDuoRecordService.queryisExitsSpiderTemp(guoDuoInfoMap);
      	    		if(null == queryisExitsSpiderTemp){
      	    			cacheHandler.hSet(TV_INFO_SPIDER_TEMP, model.getDataid(), guoDuoInfoMap);
      	    		}
      	    	}
      	}
  	    
        //设置失效时间
        cacheHandler.setCacheTimeout(TV_INFO_SPIDER_TEMP);
        
        //更新播放量记录表
        extrasMap.put("guDuoRecordDataModel", guDuoRecordDataModel);
        simpleOverviewProcess("http://ev.guduodata.com/media/simpleOverview/"+model.getDataid(), extrasMap);
	}


	private void addRequestProcess(Page page){
		try{
			List<GuDuoExceptionRecordModel> list = supplementGuDuoRecordService.queryExceptionRequestList();
			Map<String,Object> extrasMap = new HashMap<String,Object>();
			for(GuDuoExceptionRecordModel model : list){
				Thread.sleep(3000);
				if(model.getException_url().contains("simpleDetail")){
					extrasMap.put("model", model);
					simpleDetailProcess(model.getException_url(), extrasMap);
				}else if(model.getException_url().contains("simpleOverview")){
					extrasMap.put("model", model);
					simpleOverviewProcess(model.getException_url(), extrasMap);
				}
			}
		}catch(Exception e){
			System.out.println(e.getMessage());
		}
	}
	
	
	@Override
	protected String[] getStartRequestUrls() {
		return new String[]{start_url};
	}

	@Override
	protected String getSpiderType() {
		return Constants.SpiderTaskType.TV_GUDUO;
	}

	@Override
	public String getPageDataType(Page page) {
		return "other";
	}
	
	//set集合去重，不改变原有的顺序
    public String [] distinctArr(String[] arr){
        String [] newArr = new String[]{};
        List<String> newList = new ArrayList<String>();
        Set<String> set=new HashSet<String>();
        for (String str:arr) {
            if(set.add(str)){
            	newList.add(str);
            }
        }
        newArr = newList.toArray(newArr);
		return newArr;
    }
	
	
}
